import scrapy
import sys
import os
from urllib import quote
import string

sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '..'))
from common import amazon_category


class YoutubeSpider(scrapy.Spider):
    name = 'youtube'
    url = 'https://www.youtube.com/results?search_query='

    def __init__(self, *k, **kw):
        super(scrapy.Spider, self).__init__(*k, **kw)
        self.category = amazon_category.read_xlsx()

    def start_requests(self):
        #category = amazon_category.read_xlsx()
        limit =2
        print type(self.category)
        for cate in self.category:
            yield scrapy.Request(url=self.url + quote(cate), callback=self.parse)
            limit -= 1
            if limit == 0:
                break

    def parse(self, response):
        for r in response.css('.yt-lockup-content'):
            try:
                title = r.css('.yt-lockup-title a::attr(title)').extract_first(),
                url = r.css('.yt-lockup-byline a::attr(href)').extract_first(),
                try:
                    views = r.css('.yt-lockup-meta .yt-lockup-meta-info li::text')[1].extract()
                except Exception as e:
                    views = 0
            except Exception as e:
                print e
                return

            info = {
                    'title': title[0],
                    'url': url[0],
                    'views': views
            }
            # yield info
            if info['url']:
                yield scrapy.Request(response.urljoin(info['url'] + '/videos'), callback=lambda response, info=info: self.parser(response, info))

    def parser(self, response, info):
        cate_map = {}
        for r in response.css('.yt-lockup-title'):
            title = r.css('a::text').extract_first()
            c_map = self.find_category(title)
            if c_map:
                for c in c_map:
                    cate_map[c] = cate_map.get(c, 0) + c_map[c]
        cate = None
        max_cate = 0
        for c in cate_map:
            if cate_map[c] > max_cate:
                max_cate = cate_map[c]
                cate = c
        print cate_map
        if cate:
            print cate, self.category[cate]


    def find_category(self, title):
        cate_map = {}
        for cate in self.category:
            if cate in title:
                cate_map[cate] = cate_map.get(cate, 0) + 1
        return cate_map
